#ifndef POSTING_H
#define POSTING_H
#include "configure.h"
#include <boost/filesystem.hpp>
#include "lexicon.h"
#include <queue>
#include <vector>
#include <algorithm>
#include <utility>
#include "block.h"

namespace bfs = boost::filesystem;
using namespace std;
class BlockReader
{
    private:
    string blockname;
    char* buf;
    BlockRecord* recordptr;
    int recordnum;
    size_t bufsize;
    FILE* blockfid;
    size_t blocksize;
    size_t leftbytes;
    public:
    BlockReader(string _blockname,size_t _bufsize)
    {
        cout <<"blockreader constructor " << endl;
        blockname = _blockname;
        bufsize = _bufsize;
        struct stat statbuf;
        stat(blockname.c_str(),&statbuf);
        leftbytes = blocksize = statbuf.st_size;
        blockfid = fopen(blockname.c_str(),"r");
        buf = new char[bufsize];
        recordnum = 0;
    }
    BlockRecord* readrecord()
    {
        if(recordnum == 0)
        {
            if(blockfid == NULL)
                return NULL;
            if(leftbytes == 0)
            {
                fclose(blockfid);
                blockfid = NULL;
                cout << blockname << " closed" << endl;
                return NULL;
            }
            recordnum = fread(buf,sizeof(BlockRecord),bufsize / sizeof(BlockRecord),blockfid);
            leftbytes -= recordnum * sizeof(BlockRecord);
            recordptr = reinterpret_cast<BlockRecord*>(buf);
            cout << "read " << recordnum << "records from " << blockname << endl;
        }
        --recordnum;
        return recordptr++;
    }
    ~BlockReader()
    {
        cout << " blockreader deconstructor " << endl;
    }
    void free()
    {
        cout << " free memory of blockreader " << endl;
        delete[] buf;
    }
};
class PostingBuffer
{
    private:
    int  curterm;
    int* imageidbuf;
    int* imageidptr;
    int* termfqbuf;
    int* termfqptr;
    float* hashsiftbuf;
    float* hashsiftptr;
    float* mm;
    float* hashclust;
    public:
    int recordnum;
    int postingnum;
    PostingBuffer(int maxnum)
    {
        cout << " postingbuffer constructor " << endl;
        imageidptr = imageidbuf = new int[maxnum];
        termfqptr = termfqbuf = new int[maxnum];
        hashsiftptr = hashsiftbuf = new float[HAMMING_DIM*maxnum];
        mm = new float[maxnum];
        hashclust = new float[LEXICON_SIZE*HAMMING_DIM];
        init_hashclust(hashclust);
        recordnum = 0;
        postingnum = 0;
        curterm = 0;
    }
    ~PostingBuffer()
    {
        cout << " postingbuffer destructor " << endl;
    }
    void init_hashclust(float* &hashclust)
    {
        float* clust = new float[LEXICON_SIZE*SIFT_DIM];
        dense2D<float> clustmatrix(LEXICON_SIZE,SIFT_DIM);
        Quantizer::load_mat(CLUST_PATH,clust);
        for(int i=0, k=0; i<LEXICON_SIZE; ++i)
            for(int j=0; j<SIFT_DIM; ++j,++k)
                clustmatrix[i][j] = clust[k];
        dense2D<float> hamatrix(SIFT_DIM,HAMMING_DIM);
        Quantizer::load_hamatrix(HAMATRIX_PATH,hamatrix);
        dense2D<float> hashclustmatrix(LEXICON_SIZE,HAMMING_DIM);
        hashclustmatrix = clustmatrix * hamatrix;
        for(int i=0,k=0; i<LEXICON_SIZE; ++i)
            for(int j=0; j<HAMMING_DIM; ++j,++k)
                hashclust[k] = hashclustmatrix[i][j];
    }
    void write_hashclust(string filename)
    {
        cout << "write hashclust to file " << filename << endl;
        FILE* fid = fopen(filename.c_str(),"w");
        int row = LEXICON_SIZE, col = HAMMING_DIM;
        fwrite(reinterpret_cast<char*>(&row),sizeof(int),1,fid);
        fwrite(reinterpret_cast<char*>(&col),sizeof(int),1,fid);
        fwrite(reinterpret_cast<char*>(hashclust),sizeof(float),row*col,fid);
        fclose(fid);
    }
    void free()
    {
        delete[] imageidbuf;
        delete[] termfqbuf;
        delete[] hashsiftbuf;
        delete[] mm;
        delete[] hashclust;
    }
    size_t flush(string postingfilename)
    {
       //cout << "flush term " << curterm <<" to " << postingfilename << endl;
        if(postingnum == 0) return 0;
        for(int i=0,k=curterm*HAMMING_DIM; i<HAMMING_DIM; ++i,++k)
        {
            float* pp = hashsiftbuf + i;
            for(int j=0;j<recordnum; ++j,pp+=HAMMING_DIM)
            mm[j] = *pp;
            nth_element(mm,mm+(recordnum>>1),mm+recordnum);
            hashclust[k] = mm[recordnum>>1];
        }
        int* intptr = reinterpret_cast<int*>(hashsiftbuf);
        char* charptr = reinterpret_cast<char*>(hashsiftbuf)+2*sizeof(int);
        hashsiftptr = hashsiftbuf;
        char ch = 0;
        for(int i=0; i<postingnum; ++i)
        {
            int j,k,s,p;
            for(j=0; j<termfqbuf[i]; ++j)
            {
                for(k=0,p=HAMMING_DIM*curterm; k<HAMMING_DIM>>3; ++k)
                {
                    ch = 0;
                    for(s=0; s<8; ++s,++hashsiftptr,++p)
                    {
                        if(*hashsiftptr >= hashclust[p])
                        ch |= (1<<s);
                    }
                    *(charptr++) = ch;
                }
            }
            *(intptr++) = imageidbuf[i];
            *(intptr++) = termfqbuf[i];
            intptr = reinterpret_cast<int*>(charptr);
            charptr += 2*sizeof(int);
        }
        size_t writesize = reinterpret_cast<char*>(intptr) - reinterpret_cast<char*>(hashsiftbuf);
        FILE* fid = fopen(postingfilename.c_str(),"a+");
        fwrite(reinterpret_cast<char*>(hashsiftbuf),sizeof(char),writesize,fid);
        fclose(fid);
        clear();
        //cout << "flush complete " << endl;
        return writesize;
    }
    void clear()
    {
        recordnum = 0;
        postingnum = 0;
        imageidptr = imageidbuf;
        termfqptr = termfqbuf;
        hashsiftptr = hashsiftbuf;
    }
    void push(BlockRecord* br)
    {
        if(recordnum == 0 || *(imageidptr-1) != br->imageid)
        {
            *(imageidptr++) = br->imageid;
            *(termfqptr++) = 1;
            memcpy(hashsiftptr,br->hashsift,HAMMING_DIM*sizeof(float));
            hashsiftptr += HAMMING_DIM;
            ++postingnum;
        }
        else
        {
            ++(*(termfqptr-1));
            memcpy(hashsiftptr,br->hashsift,HAMMING_DIM*sizeof(float));
            hashsiftptr += HAMMING_DIM;
        }
        curterm = br->termid;
        ++recordnum;
    }
};
struct heapentry
{
    int readerid;
    BlockRecord* brptr;
    bool operator()(heapentry &a,  heapentry &b)
    {
        if((a.brptr)->termid == (b.brptr)->termid)
            return (a.brptr)->imageid > (b.brptr)->imageid;
        return (a.brptr)->termid > (b.brptr)->termid;
    }
};
int makepostingfiles()
{
    vector<string>blockname;
    bfs::path blockdir(BLOCK_PATH);
    for(bfs::directory_iterator dit(blockdir); dit!= bfs::directory_iterator(); ++dit)
    {
        if(!bfs::is_regular_file(*dit))continue;
        blockname.push_back(dit->string());
    }
    sort(blockname.begin(),blockname.end());
    int blocknum = blockname.size();
    /*
    for(int i=0; i<blocknum; ++i)
    {
        if(!checkblock(blockname[i]))
        {
            cout << "block error " << endl;
            while(true);
        }
    }
    */
    BlockReader** blockreader = new BlockReader* [blocknum];
    vector<char>emptymark(blocknum,'n');

    size_t reader_size = BLOCKREADER_BUFFER_SIZE / blocknum;
    reader_size -= reader_size % sizeof(BlockRecord);
    for(int i=0; i<blocknum; ++i)
        blockreader[i] = new BlockReader(blockname[i],reader_size);
    //利用heap多路合并block文件
    priority_queue<heapentry,vector<heapentry>,heapentry> brheap;
    heapentry entry;
    heapentry topentry;
    for(int i=0; i<blocknum; ++i)
    {
        entry.readerid = i;
        entry.brptr = blockreader[i]->readrecord();
        brheap.push(entry);
    }
    PostingBuffer pb(MAXRECORD_PER_POSTINGFILE);
    Lexicon lex;
    int postingfile_count = 0;
    int docfq = 0;
    size_t startbyte = 0;
    size_t bytenum = 0;
    int preterm = 0;
    int recordnum = 0;
    string postingfilename;
    char name[100];
    sprintf(name,"%spostingfile%06d",POSTING_PATH.c_str(),postingfile_count);
    postingfilename = string(name);

    while(!brheap.empty())
    {
        topentry = brheap.top();
        brheap.pop();
        //cout << "< " <<topentry.readerid << "," <<topentry.brptr->termid << ", " << topentry.brptr->imageid << "> " << endl;
        if((topentry.brptr)->termid != preterm)
        {
            if(recordnum + pb.recordnum > MAXRECORD_PER_POSTINGFILE)
            {
                startbyte = 0;
                ++postingfile_count;
                sprintf(name,"%spostingfile%06d",POSTING_PATH.c_str(),postingfile_count);
                postingfilename = string(name);
                cout << postingfilename;
            }
            docfq = pb.postingnum;
            bytenum = pb.flush(postingfilename);
            lex.push(postingfile_count,docfq,startbyte,bytenum);
            startbyte += bytenum;
            preterm = (topentry.brptr)->termid;
        }
        pb.push(topentry.brptr);
        if(emptymark[topentry.readerid] == 'n')
        {
            entry.readerid = topentry.readerid;
            entry.brptr = blockreader[topentry.readerid]->readrecord();
            if(entry.brptr != NULL)
                brheap.push(entry);
            else emptymark[entry.readerid] = 'y';
        }
    }
    if(recordnum + pb.recordnum > MAXRECORD_PER_POSTINGFILE)
    {
        startbyte = 0;
        ++postingfile_count;
        sprintf(name,"%spostingfile%06d",POSTING_PATH.c_str(),postingfile_count);
        postingfilename = string(name);
        cout <<postingfilename << endl;
    }
    docfq = pb.postingnum;
    bytenum = pb.flush(postingfilename);
    lex.push(postingfile_count,docfq,startbyte,bytenum);
    lex.tofile(POSTING_PATH+"lexicon");
    pb.write_hashclust(HASHCLUST_PATH);
    for(int i=0; i<blocknum; ++i)
    {
        blockreader[i]->free();
        delete blockreader[i];
    }
    delete blockreader;
    lex.free();
    pb.free();
    return postingfile_count;
}
#endif
